!pip install statsmodels
!pip install vl-convert-python
!pip install altair
!pip install pygam

d = pd.read_csv("medias-month-breakdown.csv",parse_dates=["month"])

d.shape

d.head(1)

d.value_counts("media")

alt.Chart(d[d["media"]=="liberation"]).mark_line(point=False).encode(
    x=alt.X("month:T", title="Month"),
    y=alt.Y("query_biodiversite:Q", title="Count"),
    tooltip=["month", "query_durable","total"]
).properties(
    title="Count over Time",
    width=600,
    height=300
)

media = ['lemonde', 'lesechos', 'lacroix', 'liberation', 'leparisien','lefigaro', 'mediapart']
i = 6

chart = alt.Chart(d[d["media"]==media[i]]).mark_line(point=False).encode(
    x=alt.X("month:T", title="Month"),
    y=alt.Y("total:Q", title="Count"),
    tooltip=["month", "query_durable","total"]
).properties(
    title="Total over Time - "+media[i],
    width=600,
    height=300
)

# horizontal line at y = 1000
hline = alt.Chart(pd.DataFrame({'y': [500,1000]})).mark_rule(color='red', strokeDash=[5,3]).encode(
    y='y'
)

chart+hline

# wide to long
dm = d.melt(id_vars=["month","media","total"], var_name="query", value_name="count")

# tidy queries so they are just the query term cap'd
dm["query"] = dm["query"].str[6:].str.capitalize().str.replace("_"," ")

# new columns 'normalized' and 'as' for arcsin-sqrt
dm["normalized"] = dm["count"]/dm["total"]
dm["as"] = np.arcsin(np.sqrt(dm["normalized"]))

# have a look (dl for "liberation")
dl = dm[(dm["media"]=="liberation") & (dm["query"]=="Climatique")].copy()

dl

dm.value_counts("query")

alt.Chart(dm).mark_bar().encode(
    x=alt.X("normalized:Q", bin=alt.Bin(maxbins=100), title="monthly fractions"),
    y=alt.Y("count()", title="Count")
).properties(
    title="Histogram of monthly fractions",
    width=500,
    height=300
)

# asin-sqrt -- the 0's will mostly go away as they are associated with 0 totals

alt.Chart(dm).mark_bar().encode(
    x=alt.X("as:Q", bin=alt.Bin(maxbins=100), title="monthly fractions"),
    y=alt.Y("count()", title="Count")
).properties(
    title="Histogram of monthly fractions (asin-sqrt)",
    width=500,
    height=300
)

# normalized plot 

alt.Chart(dm[(dm["month"]>="1985-01-01") & (dm["media"]=="lemonde")]).mark_line(point=False).encode(
    x=alt.X("month:T", title="Month"),
    y=alt.Y("normalized:Q", title="Fraction"),
    tooltip=["month", "query","total"]
).facet(
    facet="query:N",
    columns=3  
).properties(
    title="Count over Time",
  #  width=600,
  #  height=300
)

# same data asinsqrt transform

alt.Chart(dm[(dm["month"]>="1985-01-01") & (dm["media"]=="liberation")]).mark_line(point=False).encode(
    x=alt.X("month:T", title="Month"),
    y=alt.Y("as:Q", title="Fraction"),
    tooltip=["month", "query","total"]
).facet(
    facet="query:N",
    columns=3  
).properties(
    title="Count over Time (asin-sqrt)",
  #  width=600,
  #  height=300
)

# now a grid over the terms with different lines for the publiations

alt.Chart(dm[(dm["month"]>="1985-01-01") ]).mark_line(point=False).encode(
    x=alt.X("month:T", title="Month"),
    y=alt.Y("as:Q", title="Fraction"),
    tooltip=["month", "query","total"],
    color="media"
).facet(
    facet="query:N",
    columns=3  
).properties(
    title="Count over Time (asin-sqrt)",
  #  width=600,
  #  height=300
)

dm.value_counts("media")

dm.value_counts("query")

# updated function to build one smoothed plot

def build_smoothed_plot(media_name, query_name, w, h, first_row=True, first_col=True):
    
    media = cleanup_media_name(media_name)
    query = query_name.replace("é", "e")
    d = dm[(dm["media"]==media) & (dm["query"]==query)].copy()
    d = d[d["month"]>="1985-01-01"]
    d = d.reset_index(drop=True)

#  the old smooth
#    d["as_loess"] = pd.Series(np.sin(smooth_keep_spikes(list(d["as"]), frac=0.05, k=4))**2,index=d.index)

#  the new smooth
    e = out["smooth"][(out["media"]==media) & (out["query"]==query)].copy()
    e = e.reset_index(drop=True)
    d["as_loess"] = e
    
    p1 = alt.Chart(d).mark_line(point=False,color="red").encode(
        x=alt.X("month:T", title=None, scale=alt.Scale(domain=["1985-01-01", "2024-12-01"])),
        y=alt.Y("as_loess:Q", title=media_name if first_col else None, scale=alt.Scale(domain=[0, 0.2])).axis(format='%'),
        tooltip=["month", "query","total"]
    ).properties(
        title=query_name if first_row else "",
        width=w,
        height=h
    )

    p2 = alt.Chart(d).mark_line(point=False,opacity=0.2).encode(
        x=alt.X("month:T", title=None, scale=alt.Scale(domain=["1985-01-01", "2024-12-01"])),
        y=alt.Y("normalized:Q", title=media_name if first_col else None, scale=alt.Scale(domain=[0, 0.2])).axis(format='%'),
        tooltip=["month", "query","total"]
    ).properties(
        title=query_name if first_row else "",
        width=w,
        height=h
    )

    return p2+p1

build_smoothed_plot("liberation", "Climatique", 600, 300)

# now make a grid of small multiples across medias/queries (same code)

medias = [
    "Médiapart",
    "Libération",
    "Le Monde",
    "La Croix",
    "Le Parisien",
    "Les Échos",
    "Le Figaro"
]
queries = [
    "Climatique",
    "Effet de serre",
    "Biodiversité",
#    "Durable",
#    "Transition"
]

small_multiples = alt.vconcat()
for i, media in enumerate(medias):
    query_charts = alt.hconcat()
    for j, query in enumerate(queries):
        print(query,media)
        query_charts |= build_smoothed_plot(media, query, 400, 125, not i, not j)
    small_multiples &= query_charts

small_multiples

media = "leparisien"

dl = dm[(dm["month"]>="1985-01-01") & (dm["media"]==media) & (dm["query"]=="Climatique")].copy()
dl["as_loess"] = pd.Series(np.sin(smooth_keep_spikes(list(dl["as"]), frac=0.05, k=4.0))**2,index=dl.index)
dl = dl.reset_index(names="x")

# Create the stripes
chart1 = alt.Chart(dl).mark_rect().encode(
        x=alt.X("x:O", axis=None),         # ordinal so stripes are equally spaced
        color=alt.Color(
            "as:Q",
            scale=alt.Scale(scheme="redblue", reverse=True),  # blue=cold, red=hot
            legend=None,
        ),
    ).properties(width=1000, height=300) # removes border

dl = dm[(dm["month"]>="1985-01-01") & (dm["media"]==media) & (dm["query"]=="Effet de serre")].copy()
dl["as_loess"] = pd.Series(np.sin(smooth_keep_spikes(list(dl["as"]), frac=0.05, k=4.0))**2,index=dl.index)
dl = dl.reset_index(names="x")

# Create the stripes
chart2 = alt.Chart(dl).mark_rect().encode(
        x=alt.X("x:O", axis=None),         # ordinal so stripes are equally spaced
        color=alt.Color(
            "as:Q",
            scale=alt.Scale(scheme="redblue", reverse=True),  # blue=cold, red=hot
            legend=None,
        ),
    ).properties(width=1000, height=300)  # removes border

dl = dm[(dm["month"]>="1985-01-01") & (dm["media"]==media) & (dm["query"]=="Durable")].copy()
dl["as_loess"] = pd.Series(np.sin(smooth_keep_spikes(list(dl["as"]), frac=0.05, k=4.0))**2,index=dl.index)
dl = dl.reset_index(names="x")

# Create the stripes
chart3 = alt.Chart(dl).mark_rect().encode(
        x=alt.X("x:O", axis=None),         # ordinal so stripes are equally spaced
        color=alt.Color(
            "as:Q",
            scale=alt.Scale(scheme="redblue", reverse=True),  # blue=cold, red=hot
            legend=None,
        ),
    ).properties(width=1000, height=300)  # removes border

dl = dm[(dm["month"]>="1985-01-01") & (dm["media"]==media) & (dm["query"]=="Biodiversite")].copy()
dl["as_loess"] = pd.Series(np.sin(smooth_keep_spikes(list(dl["as"]), frac=0.05, k=4.0))**2,index=dl.index)
dl = dl.reset_index(names="x")

# Create the stripes
chart4 = alt.Chart(dl).mark_rect().encode(
        x=alt.X("x:O", axis=None),         # ordinal so stripes are equally spaced
        color=alt.Color(
            "as:Q",
            scale=alt.Scale(scheme="redblue", reverse=True),  # blue=cold, red=hot
            legend=None,
        ),
    ).properties(width=1000, height=300)  # removes border


(chart1 & chart2 & chart4 & chart3).configure_view(strokeWidth=0).resolve_scale(color="independent")#.save(media+".png")

import pandas as pd
import altair as alt
alt.data_transformers.disable_max_rows()

data[data["media"]=="lefigaro"].head(100)

data.value_counts("media")

stripes0 = make_stripes("Médiapart",dates,data,800,100)
stripes1 = make_stripes("Libération",dates,data,800,100)
stripes2 = make_stripes("Le Monde",dates,data,800,100)
stripes3 = make_stripes("La Croix",dates,data,800,100)
stripes4 = make_stripes("Le Parisien",dates,data,800,100)
stripes5 = make_stripes("Les Échos",dates,data,800,100)
stripes6 = make_stripes("Le Figaro",dates,data,800,100)

chart_stack = (alt.vconcat(stripes1, stripes2, stripes3, stripes4, stripes5, stripes6)
    .resolve_scale(x='shared',color="independent")
)

chart_stack

timeAxis = alt.Chart(data[(data["month"]>="1985-01-01") & (data["media"]=="lemonde")]).mark_line(point=False, opacity=0).encode(
    x=alt.X("month:T", title="Date"),
    y=alt.Y("frac:O", title=None, axis=None)
).properties(
    width=800,
    height=1
)
timeAxis

alt.vconcat(chart_stack, timeAxis).configure_view(stroke=None)